{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "AFIchtHPxrM5"
   },
   "outputs": [],
   "source": [
    "\n",
    "import tensorflow as tf\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "import argparse\n",
    "import sys\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import urllib\n",
    "import time\n",
    "FLAGS = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 433
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1559,
     "status": "ok",
     "timestamp": 1529408857125,
     "user": {
      "displayName": "Lip Gallagher",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "113091702821929511633"
     },
     "user_tz": -480
    },
    "id": "1aYBNpD-zglv",
    "outputId": "cd4df2f3-10a1-4ec9-c4f7-71959d90f467"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-7e828717a4ff>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# load data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "FjT1WThmzka2"
   },
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(initial, collections=[tf.GraphKeys.GLOBAL_VARIABLES,'Weights'])\n",
    "def bias_variable(shape):\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(initial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "FwguWk_zznsZ"
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])#\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "bM-jS-lCzp-k"
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('reshape'):\n",
    "    x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "with tf.name_scope('conv1'):\n",
    "    w_conv1 = weight_variable([5,5,1,32])\n",
    "    b_conv1 = bias_variable([32])\n",
    "    l_conv1 = tf.nn.conv2d(x_image, w_conv1, strides=[1,1,1,1],\n",
    "                          padding = 'SAME') + b_conv1\n",
    "    # 激活函数\n",
    "    h_conv1 = tf.nn.relu(l_conv1)\n",
    "#output batch,28,28,32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "jBJMAAk8zr-A"
   },
   "outputs": [],
   "source": [
    "# 28*28 - 14*14(32)\n",
    "with tf.name_scope('pool1'):\n",
    "    h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1,2,2,1],\n",
    "                            strides =[1,2,2,1], padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "FSGuftHAzuA6"
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('conv2'):\n",
    "    w_conv2 = weight_variable([5,5,32,64])\n",
    "    b_conv2 = bias_variable([64])\n",
    "    l_conv2 = tf.nn.conv2d(h_pool1, w_conv2, strides=[1,1,1,1],\n",
    "                          padding='SAME') + b_conv2\n",
    "    h_conv2 = tf.nn.relu(l_conv2)\n",
    "# output batch,14,14,64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "ebphhYCszvtS"
   },
   "outputs": [],
   "source": [
    "keep_prob = tf.placeholder(tf.float32) #\n",
    "h_conv2_drop = tf.nn.dropout(h_conv2, keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "rEJLfOPbzxq-"
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('pool2'):\n",
    "    h_pool2 = tf.nn.max_pool(h_conv2_drop, ksize=[1,2,2,1],\n",
    "                            strides=[1,2,2,1], padding='SAME')\n",
    "# 14x14x64 -->  7x7x64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "hbZIZGdj-Qqz"
   },
   "outputs": [],
   "source": [
    "#with tf.name_scope('conv3'):\n",
    "#  w_conv3 = weight_variable([4,4,12,24])\n",
    "#  b_conv3 = bias_variable([24])\n",
    "#  l_conv3 = tf.nn.conv2d(h_conv2, w_conv3, strides=[1,2,2,1],\n",
    "#                        padding='SAME') + b_conv3\n",
    "#  h_conv3 = tf.nn.relu(l_conv3)\n",
    "# output batch,7,7,24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "Plh29ot5zzwu"
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('fc1'):\n",
    "    w_fc1 = weight_variable([7*7*64, 1024])\n",
    "    b_fc1 = bias_variable([1024])\n",
    "    h_pool2_flat = tf.reshape(h_pool2, [-1,7*7*64])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "wB9OGzioz5dt"
   },
   "outputs": [],
   "source": [
    "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "v7YFjNdsz7WB"
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('fc2'):\n",
    "    w_fc2 = weight_variable([1024,10])\n",
    "    b_fc2 = bias_variable([10])\n",
    "    y = (tf.matmul(h_fc1_drop, w_fc2) + b_fc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "d0YaC8aAz82C"
   },
   "outputs": [],
   "source": [
    "decay_rate = 0.96\n",
    "\n",
    "decay_steps = 1000\n",
    "\n",
    "global_ = tf.Variable(tf.constant(0))\n",
    "\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y))\n",
    "\n",
    "#l2_loss = tf.add_n([tf.nn.l2_loss(w) for w in tf.get_collection('Weights')])\n",
    "\n",
    "#total_loss = cross_entropy + 7e-5*l2_loss\n",
    "\n",
    "\n",
    "\n",
    "#updateparameter = tf.group(update_parameter, update_parameter2)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_, 1))\n",
    "\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "lr = tf.train.exponential_decay(0.001, global_, 1000,0.96 ) # (learning_rate, global, decay_step, decay_rate)\n",
    "train_step = tf.train.AdamOptimizer(lr).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 1835
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3016442,
     "status": "ok",
     "timestamp": 1529411886554,
     "user": {
      "displayName": "Lip Gallagher",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "113091702821929511633"
     },
     "user_tz": -480
    },
    "id": "KTSpS4qXz-rm",
    "outputId": "65fc8e80-c41f-4c1a-b0fc-5e4ce0def130"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100 : entropy loss: 0.248254, learning_rate: 0.000995967\n",
      "step 200 : entropy loss: 0.0950434, learning_rate: 0.000991909\n",
      "step 300 : entropy loss: 0.0421758, learning_rate: 0.000987869\n",
      "step 400 : entropy loss: 0.043535, learning_rate: 0.000983844\n",
      "step 500 : entropy loss: 0.0618557, learning_rate: 0.000979836\n",
      "step 600 : entropy loss: 0.0219418, learning_rate: 0.000975844\n",
      "step 700 : entropy loss: 0.0213289, learning_rate: 0.000971869\n",
      "step 800 : entropy loss: 0.00797945, learning_rate: 0.000967909\n",
      "step 900 : entropy loss: 0.0293583, learning_rate: 0.000963966\n",
      "step 1000 : entropy loss: 0.0783265, learning_rate: 0.000960039\n",
      "step 1100 : entropy loss: 0.0477204, learning_rate: 0.000956128\n",
      "step 1200 : entropy loss: 0.0649866, learning_rate: 0.000952233\n",
      "step 1300 : entropy loss: 0.0318687, learning_rate: 0.000948354\n",
      "step 1400 : entropy loss: 0.0233917, learning_rate: 0.00094449\n",
      "step 1500 : entropy loss: 0.0822123, learning_rate: 0.000940642\n",
      "step 1600 : entropy loss: 0.0047588, learning_rate: 0.00093681\n",
      "step 1700 : entropy loss: 0.0130411, learning_rate: 0.000932994\n",
      "step 1800 : entropy loss: 0.0175162, learning_rate: 0.000929193\n",
      "step 1900 : entropy loss: 0.0129214, learning_rate: 0.000925408\n",
      "step 2000 : entropy loss: 0.081499, learning_rate: 0.000921638\n",
      "step 2100 : entropy loss: 0.0314244, learning_rate: 0.000917883\n",
      "step 2200 : entropy loss: 0.0148242, learning_rate: 0.000914144\n",
      "step 2300 : entropy loss: 0.0631858, learning_rate: 0.00091042\n",
      "step 2400 : entropy loss: 0.0203983, learning_rate: 0.000906711\n",
      "step 2500 : entropy loss: 0.00188272, learning_rate: 0.000903017\n",
      "step 2600 : entropy loss: 0.0116906, learning_rate: 0.000899338\n",
      "step 2700 : entropy loss: 0.0640247, learning_rate: 0.000895674\n",
      "step 2800 : entropy loss: 0.00161308, learning_rate: 0.000892025\n",
      "step 2900 : entropy loss: 0.0348313, learning_rate: 0.000888391\n",
      "step 3000 : entropy loss: 0.0712715, learning_rate: 0.000884772\n",
      "step 3100 : entropy loss: 0.00129136, learning_rate: 0.000881168\n",
      "step 3200 : entropy loss: 0.00628291, learning_rate: 0.000877578\n",
      "step 3300 : entropy loss: 0.00517798, learning_rate: 0.000874003\n",
      "step 3400 : entropy loss: 0.00853432, learning_rate: 0.000870442\n",
      "step 3500 : entropy loss: 0.00180284, learning_rate: 0.000866896\n",
      "step 3600 : entropy loss: 0.00399926, learning_rate: 0.000863364\n",
      "step 3700 : entropy loss: 0.00392369, learning_rate: 0.000859847\n",
      "step 3800 : entropy loss: 0.000919922, learning_rate: 0.000856344\n",
      "step 3900 : entropy loss: 0.00509662, learning_rate: 0.000852856\n",
      "step 4000 : entropy loss: 0.00246378, learning_rate: 0.000849381\n",
      "step 4100 : entropy loss: 0.0506046, learning_rate: 0.000845921\n",
      "step 4200 : entropy loss: 0.00607352, learning_rate: 0.000842475\n",
      "step 4300 : entropy loss: 0.00471753, learning_rate: 0.000839043\n",
      "step 4400 : entropy loss: 0.0534583, learning_rate: 0.000835624\n",
      "step 4500 : entropy loss: 0.000793513, learning_rate: 0.00083222\n",
      "step 4600 : entropy loss: 0.0180178, learning_rate: 0.00082883\n",
      "step 4700 : entropy loss: 0.00803755, learning_rate: 0.000825453\n",
      "step 4800 : entropy loss: 0.00805874, learning_rate: 0.00082209\n",
      "step 4900 : entropy loss: 0.00118149, learning_rate: 0.000818741\n",
      "step 5000 : entropy loss: 0.0325416, learning_rate: 0.000815406\n",
      "step 5100 : entropy loss: 0.00609198, learning_rate: 0.000812084\n",
      "step 5200 : entropy loss: 0.051459, learning_rate: 0.000808776\n",
      "step 5300 : entropy loss: 0.00146332, learning_rate: 0.000805481\n",
      "step 5400 : entropy loss: 0.00452552, learning_rate: 0.000802199\n",
      "step 5500 : entropy loss: 0.0104858, learning_rate: 0.000798931\n",
      "step 5600 : entropy loss: 0.0215469, learning_rate: 0.000795677\n",
      "step 5700 : entropy loss: 0.00424908, learning_rate: 0.000792435\n",
      "step 5800 : entropy loss: 0.0575134, learning_rate: 0.000789207\n",
      "step 5900 : entropy loss: 0.00964909, learning_rate: 0.000785992\n",
      "step 6000 : entropy loss: 0.00281321, learning_rate: 0.00078279\n",
      "step 6100 : entropy loss: 0.00402138, learning_rate: 0.000779601\n",
      "step 6200 : entropy loss: 0.00645867, learning_rate: 0.000776425\n",
      "step 6300 : entropy loss: 0.000295022, learning_rate: 0.000773262\n",
      "step 6400 : entropy loss: 0.000274069, learning_rate: 0.000770111\n",
      "step 6500 : entropy loss: 0.001577, learning_rate: 0.000766974\n",
      "step 6600 : entropy loss: 0.0033479, learning_rate: 0.000763849\n",
      "step 6700 : entropy loss: 0.000508139, learning_rate: 0.000760738\n",
      "step 6800 : entropy loss: 0.0025993, learning_rate: 0.000757638\n",
      "step 6900 : entropy loss: 0.00113286, learning_rate: 0.000754552\n",
      "step 7000 : entropy loss: 0.0168018, learning_rate: 0.000751478\n",
      "step 7100 : entropy loss: 0.00397222, learning_rate: 0.000748417\n",
      "step 7200 : entropy loss: 0.0122369, learning_rate: 0.000745368\n",
      "step 7300 : entropy loss: 0.00527804, learning_rate: 0.000742331\n",
      "step 7400 : entropy loss: 9.11518e-05, learning_rate: 0.000739307\n",
      "step 7500 : entropy loss: 4.85053e-05, learning_rate: 0.000736295\n",
      "step 7600 : entropy loss: 0.00704465, learning_rate: 0.000733296\n",
      "step 7700 : entropy loss: 0.013639, learning_rate: 0.000730308\n",
      "step 7800 : entropy loss: 0.0127144, learning_rate: 0.000727333\n",
      "step 7900 : entropy loss: 5.81461e-05, learning_rate: 0.00072437\n",
      "step 8000 : entropy loss: 0.00105841, learning_rate: 0.000721419\n",
      "step 8100 : entropy loss: 0.000326534, learning_rate: 0.00071848\n",
      "step 8200 : entropy loss: 0.000123349, learning_rate: 0.000715553\n",
      "step 8300 : entropy loss: 0.00413006, learning_rate: 0.000712638\n",
      "step 8400 : entropy loss: 0.000781728, learning_rate: 0.000709735\n",
      "step 8500 : entropy loss: 0.000291447, learning_rate: 0.000706843\n",
      "step 8600 : entropy loss: 0.00493428, learning_rate: 0.000703964\n",
      "step 8700 : entropy loss: 4.10058e-06, learning_rate: 0.000701096\n",
      "step 8800 : entropy loss: 6.60892e-05, learning_rate: 0.00069824\n",
      "step 8900 : entropy loss: 8.62555e-05, learning_rate: 0.000695395\n",
      "step 9000 : entropy loss: 0.000115377, learning_rate: 0.000692562\n",
      "step 9100 : entropy loss: 0.0200043, learning_rate: 0.000689741\n",
      "step 9200 : entropy loss: 0.000607193, learning_rate: 0.000686931\n",
      "step 9300 : entropy loss: 4.59706e-05, learning_rate: 0.000684132\n",
      "step 9400 : entropy loss: 0.000910848, learning_rate: 0.000681345\n",
      "step 9500 : entropy loss: 0.000990818, learning_rate: 0.00067857\n",
      "step 9600 : entropy loss: 0.00456757, learning_rate: 0.000675805\n",
      "step 9700 : entropy loss: 0.000360393, learning_rate: 0.000673052\n",
      "step 9800 : entropy loss: 0.00380879, learning_rate: 0.00067031\n",
      "step 9900 : entropy loss: 9.13246e-05, learning_rate: 0.000667579\n",
      "step 10000 : entropy loss: 0.00377107, learning_rate: 0.00066486\n",
      "test accuracy 0.9901\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    for step in range(10000):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        ts, loss, l, = sess.run([train_step, cross_entropy, lr],\n",
    "                                 feed_dict={x:batch_xs, y_:batch_ys, keep_prob: 0.75, global_: step})\n",
    "        test_acc = sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                            y_: mnist.test.labels, keep_prob:1.0})\n",
    "        if (step+1) % 100==0:\n",
    "            print('step %d : entropy loss: %g, learning_rate: %g' %(step+1, loss, l))\n",
    "    print('test accuracy %g' % (test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "TseGjjjW0Uv_"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "default_view": {},
   "name": "Untitled0.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
